package cn.lyjuan.first.hadoop.demo.ch04;

import cn.lyjuan.base.util.RandomUtils;
import cn.lyjuan.first.hadoop.demo.enums.ChNameEnum;
import cn.lyjuan.first.hadoop.util.FileUtil;
import cn.lyjuan.first.hadoop.util.HDFSUtil;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;

import java.io.IOException;
import java.util.ArrayList;
import java.util.List;

/**
 * 利用Hadoop自带功能进行数字升序排序
 */
public class Ch04S04Sort {

    /**
     * 数据处理任务:<br/>
     * 1. 直接输出数据<br/>
     * <p>
     * Notes: Mapper输入跟Reduce输入输出必须一致
     * </p>
     */
    public static class SortMapper extends Mapper<LongWritable, Text, IntWritable, IntWritable> {
        private static IntWritable data = new IntWritable();
        private static IntWritable DEF_VALUE = new IntWritable(1);

        @Override
        protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
            data.set(Integer.parseInt(value.toString()));
            context.write(data, DEF_VALUE);
        }
    }

    /**
     * 数据合并、排序<br/>
     * 利用 Mapper输出Key为使用Hadoop排序功能，默认为升序
     */
    public static class SortReduce extends Reducer<IntWritable, IntWritable, IntWritable, IntWritable> {
        /**
         * 利用静态变量保存行号
         */
        private static IntWritable lineNum = new IntWritable(0);

        @Override
        protected void reduce(IntWritable key, Iterable<IntWritable> values, Context context) throws IOException, InterruptedException {
            for (IntWritable n : values) {
                context.write(lineNum, key);
                lineNum.set(lineNum.get() + 1);
            }
        }
    }

    public static final Path OUT_PATH = FileUtil.remoteURIPath(ChNameEnum.CH04, Ch04S04Sort.class, "result");

    public static final List<Path> inFiles = new ArrayList<>();

    /**
     * 生成HDFS源数据文件
     */
    public static void loadInputFiles() {
        int fileCount = RandomUtils.randomInt(2);
        Path remoteDir = new Path(FileUtil.remotePath(ChNameEnum.CH04, Ch04S04Sort.class));
        HDFSUtil.del(remoteDir);// 清除旧数据
        Path path = null;
        for (int i = 0; i < fileCount; i++) {
            path = FileUtil.remoteURIPath(ChNameEnum.CH04, Ch04S04Sort.class, String.valueOf(i));
            inFiles.add(path);
            int row = RandomUtils.randomInt(2);
            StringBuilder sb = new StringBuilder();
            for (int j = 0; j < row; j++) {
                sb.append("\n").append(RandomUtils.randomInt(4));
            }
            sb.deleteCharAt(0);
            HDFSUtil.writeRemote(path, sb.toString());
            System.out.println("write remote file: " + path.toString());
        }
    }

    public static void main(String[] args) throws Exception {
        // 加载源文件
        loadInputFiles();

//        HDFSUtil.conf().set("mapred.job.tracker", "192.168.1.200:9001");

        Job job = new Job(HDFSUtil.conf(), "sort");
        job.setUser(HDFSUtil.USER);
        job.setJarByClass(Ch04S04Sort.class);
        job.setMapperClass(SortMapper.class);

//        job.setCombinerClass(AngleReduce.class);
        job.setReducerClass(SortReduce.class);
        job.setOutputKeyClass(IntWritable.class);
        job.setOutputValueClass(IntWritable.class);
        for (Path i : inFiles)
            FileInputFormat.addInputPath(job, i);
        FileOutputFormat.setOutputPath(job, OUT_PATH);
        boolean isOk = job.waitForCompletion(true);
        System.out.println("result ==> " + isOk);
        if (!isOk) return;
        // 成功时生成结果文件，报错时不生成任何文件
        FileStatus[] results = HDFSUtil.ls(OUT_PATH);
        for (FileStatus f : results) {
            System.out.println("====== " + f.getPath().getName() + " ======");
            System.out.println(new String(HDFSUtil.readRemote(f.getPath())));
            System.out.println();
            System.out.println();
        }
    }
}
